In [1]:
#%matplotlib notebook
import os
import matplotlib.pyplot as plt
import numpy as np
from JSAnimation import IPython_display
from matplotlib import animation
import matplotlib.patches as mpatches
import cv2
import imageio
import mpld3
import scipy.misc
from RL_utils import *
import tensorflow as tf
import tensorflow.contrib.layers as c_layers
import seaborn as sns
import pandas as pd
import glob
from scipy import misc
In [2]:
folder_name = '3999_16.100'
path = './Results/TowerTraining/'+folder_name+'/'
obs = np.load(path+'visobs.npy', mmap_mode='r')
vec = np.load(path+'vecobs.npy')
enc = np.load(path+'encodings.npy')
val = np.load(path+'values.npy')
act = np.load(path+'actions.npy')
rew = np.load(path+'rewards.npy')
doors = np.load(path+'doors.npy')
Rewards = np.load(path+'reward_per.npy')
hand_l = pd.read_csv(path+'HandLabels.csv')
label_test = np.zeros(enc.shape[0])
label_test = np.array(hand_l['Label'])
Get Random Agent Activations¶
In [3]:
tf.reset_default_graph()
def swish(input_activation):
"""Swish activation function. For more info: https://arxiv.org/abs/1710.05941"""
return tf.multiply(input_activation, tf.nn.sigmoid(input_activation))
def create_global_steps():
"""Creates TF ops to track and increment global training step."""
global_step = tf.Variable(0, name="global_step", trainable=False, dtype=tf.int32)
increment_step = tf.assign(global_step, tf.add(global_step, 1))
return global_step, increment_step
global_step, increment_step = create_global_steps()
o_size_h = 168
o_size_w = 168
vec_obs_size = 8
num_layers = 2
h_size = 256
h_size_vec = 256
visual_in = tf.placeholder(shape=[None, o_size_h, o_size_w, 3], dtype=tf.float32,name="visual_observation_0")
vector_in = tf.placeholder(shape=[None, vec_obs_size], dtype=tf.float32,name="vector_observation_0")
running_mean = tf.get_variable("running_mean", [vec_obs_size],trainable=False, dtype=tf.float32,initializer=tf.zeros_initializer())
running_variance = tf.get_variable("running_variance", [vec_obs_size],trainable=False,dtype=tf.float32,initializer=tf.ones_initializer())
mean_current_observation = tf.reduce_mean(vector_in, axis=0)
new_mean = running_mean + (mean_current_observation - running_mean) / tf.cast(tf.add(global_step, 1), tf.float32)
new_variance = running_variance + (mean_current_observation - new_mean) * (mean_current_observation - running_mean)
update_mean = tf.assign(running_mean, new_mean)
update_variance = tf.assign(running_variance, new_variance)
normalized_state = tf.clip_by_value((vector_in - running_mean) / tf.sqrt(running_variance / (tf.cast(global_step, tf.float32) + 1)), -5, 5,name="normalized_state")
def create_vector_observation_encoder(observation_input, h_size, activation, num_layers, scope,reuse):
with tf.variable_scope(scope):
hidden_vec = observation_input
for i in range(num_layers):
hidden_vec = tf.layers.dense(hidden_vec, h_size, activation=activation, reuse=reuse,name="hidden_{}".format(i),kernel_initializer=c_layers.variance_scaling_initializer(1.0))
return hidden_vec
def create_visual_observation_encoder(image_input, h_size, activation, num_layers, scope,reuse):
with tf.variable_scope(scope):
conv1 = tf.layers.conv2d(image_input, 16, kernel_size=[8, 8], strides=[4, 4],activation=tf.nn.elu, reuse=reuse, name="conv_1")
conv2 = tf.layers.conv2d(conv1, 32, kernel_size=[4, 4], strides=[2, 2],activation=tf.nn.elu, reuse=reuse, name="conv_2")
hidden_vis = c_layers.flatten(conv2)
with tf.variable_scope(scope + '/' + 'flat_encoding'):
hidden_flat = create_vector_observation_encoder(hidden_vis, h_size, activation,num_layers, scope, reuse)
return hidden_flat
visual_encoders = []
hidden_state, hidden_visual = None, None
encoded_visual = create_visual_observation_encoder(visual_in,h_size,swish,num_layers,"main_graph_0_encoder0", False)
visual_encoders.append(encoded_visual)
hidden_visual = tf.concat(visual_encoders, axis=1)
hidden_state = create_vector_observation_encoder(vector_in,h_size_vec, swish,num_layers,"main_graph_0",False)
final_hidden = tf.concat([hidden_visual, hidden_state], axis=1)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
random_enc = sess.run(final_hidden, feed_dict={visual_in: obs, vector_in: vec[0]})
Data Preparation¶
In [4]:
encodings = np.zeros((obs.shape[0],16,32))
for i in range(obs.shape[0]):
encodings[i] = enc[i].reshape((16,32))
enc2D = enc.reshape(enc.shape[0],enc.shape[-1])
print(enc2D.shape)
R_encodings = np.zeros((obs.shape[0],16,32))
for i in range(obs.shape[0]):
R_encodings[i] = random_enc[i].reshape((16,32))
R_enc2D = random_enc.reshape(random_enc.shape[0],random_enc.shape[-1])
print(R_enc2D.shape)
Run Statistics¶
In [65]:
fig = plt.figure(figsize=(30,10))
ax = plt.subplot(1,2,1)
plt.plot(vec[0][:,7],linewidth=7)
plt.plot(rew,linewidth=5)
plt.plot(val[:,:,0],linewidth=5)
plt.legend(['Level','Reward','Value Estimate'], fontsize=30)
plt.title('Inference Run - Agent Performance', fontsize=40)
plt.xlabel('Steps',fontsize=30)
plt.rc('xtick',labelsize=20)
plt.rc('ytick',labelsize=20)
plt.subplot(1,2,2)
plt.title('Agent Observations\n',fontsize=40)
image = misc.imread(path+'figures/ExampleObs.jpg')
plt.imshow(image)
plt.axis('off')
#plt.show()
#mpld3.save_html(fig, 'myfig.html')
plt.savefig(path+'figures/TrainStatistics.eps', bbox_inches='tight', format='eps')
plt.savefig(path+'figures/TrainStatistics.png', bbox_inches='tight')
In [6]:
fig = plt.figure(figsize=(15,10))
#ax = plt.subplot(1,2,1)
plt.plot(vec[0][:,7],linewidth=8)
plt.plot(val[:,:,0],linewidth=5)
plt.plot(rew,linewidth=3)
plt.legend(['Level','Value Estimate','Reward'], fontsize=30)
plt.title('Inference Run - Agent Performance', fontsize=40)
plt.xlabel('Steps',fontsize=30)
plt.rc('xtick',labelsize=20)
plt.rc('ytick',labelsize=20)
plt.xlim(0,4000)
plt.ylim(0,11)
plt.savefig(path+'figures/TrainStatistics2.eps', bbox_inches='tight', format='eps')
plt.savefig(path+'figures/TrainStatistics2.png', bbox_inches='tight')
#plt.show()
In [7]:
print('Reward is received in '+str(rew[rew>0].shape[0])+' out of '+str(rew.shape[0])+' frames ('+str(rew[rew>0].shape[0]/rew.shape[0]*100)+'%).')
print('Only '+str(rew[rew>0.9].shape[0])+' of these contain a full reward of 1.')
In [251]:
f = plot_actions(act)
#plt.show()
plt.savefig(path+'figures/ActionDistributions.eps', bbox_inches='tight')
Get Action Combination at Each Frame¶
In [22]:
def where_array_equal(a,b):
equal = np.zeros(a.shape[0])
for i,arr in enumerate(a):
if np.array_equal(arr,b):
equal[i] = 1
return equal
w = where_array_equal(act[:,0],np.unique(act[:,0],axis=0)[3])
In [23]:
combinations = np.unique(act[:,0],axis=0)
a_vec = np.zeros((combinations.shape[0],act.shape[0]))
for i,c in enumerate(combinations):
a_vec[i] = where_array_equal(act[:,0],c)
print(a_vec.shape)
Merge Combinations¶
In [24]:
all_a_comb = np.zeros(act.shape[0])
for i in range(act.shape[0]):
if a_vec[0][i] == 1:
all_a_comb[i] = 0#[1, 0, 0, 1] Forward, Turn right
elif a_vec[3][i] == 1:
all_a_comb[i] = 1#[1, 1, 0, 1] Forward, Camera Left, Turn Right
elif a_vec[4][i] == 1:
all_a_comb[i] = 2#[1, 1, 0, 2] Forward, Camera Left, Turn Left
elif a_vec[6][i] == 1:
all_a_comb[i] = 3#[1, 2, 0, 1] Forward, Camera Right, Turn Right
elif a_vec[13][i] == 1:
all_a_comb[i] = 4#[2, 2, 0, 1] Backward, Camera Right, Turn Right
else:
all_a_comb[i] = 5#Anything else
In [107]:
action_C = ['Forward,\nTurn Right','Forward,\nCamera Left,\nTurn Right','Forward,\nCamera Left,\nTurn Left',
'Forward,\nCamera Right,\nTurn Right','Backward,\nCamera Right,\nTurn Right','Everything\nElse']
plt.figure(figsize=(10,15))
plt.subplots_adjust(hspace=0.2,wspace=0.01)
ax1 = plt.subplot(2,1,1)
plt.title("Distribution of Frames for all Action Combinations",fontsize=20)
p1 = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,7,7,1,3,7,4,7,7,7,7,7,7,6,7])]
b1 = sns.barplot(x=np.linspace(0,15,16),y=np.sum(a_vec,axis=1),palette=p1)
plt.xlabel("Action Combination",fontsize=15)
plt.ylabel("# of Frames",fontsize=15)
plt.xticks(np.linspace(0,15,16),np.linspace(0,15,16,dtype=int))
#plt.yscale('log')
ax2 = plt.subplot(2,1,2)
plt.title("Distribution of Frames for Selected Action Combinations",fontsize=20)
p2 = np.array(sns.color_palette("Accent", n_colors=8))[np.array([0,1,3,4,6,7])]
a2 = np.histogram(all_a_comb,bins=np.linspace(-0.5,5.5,7))
b2 = sns.barplot(x=np.linspace(0,5,6),y=a2[0],palette=p2,ax=ax2)
plt.xlabel("Action Combination",fontsize=15)
plt.ylabel("# of Frames",fontsize=15)
plt.xticks(np.linspace(0,5,6),action_C)
#plt.show()
plt.savefig(path+'figures/ActionCombDist.eps', bbox_inches='tight')
Agent State Representation¶
In [10]:
end = 200
plot_movie_jsInfo(encodings[:end],obs[:end],act[:end],val[:end],rew[:end])#,save=path+'encodingsInfo.mp4')